{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b73f7b37",
   "metadata": {},
   "source": [
    "### 初始化环境"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ae1bc682",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 在jupyter notebook里env.render看不到窗口\n",
    "# 写一个helper类，用matplotlib刷新显示图像\n",
    "# 初始化传入env，调用helper的render即可\n",
    "from IPython import display # 导入display模块，用于在Jupyter Notebook中显示图像\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt # 导入matplotlib库，用于绘制图像\n",
    "%matplotlib inline\n",
    "\n",
    "class GymHelper:\n",
    "    def __init__(self, env, figsize = (3, 3)):\n",
    "        self.env = env # 初始化Gym环境\n",
    "        self.figsize = figsize # 初始化绘图窗口大小\n",
    "        \n",
    "        plt.figure(figsize = figsize) # 创建绘图窗口\n",
    "        plt.title(self.env.spec.id) # 标题设为环境名\n",
    "        self.img = plt.imshow(env.render()) # 在绘图窗口中显示初始图像\n",
    "    \n",
    "    def render(self, title = None):\n",
    "        image_data = self.env.render() # 获取当前环境图像渲染数据\n",
    "        \n",
    "        self.img.set_data(image_data) # 更新绘图窗口中的图像数据\n",
    "        display.display(plt.gcf()) # 刷新显示\n",
    "        display.clear_output(wait = True) # 有新图片时再清除绘图窗口原有图像\n",
    "        if title: # 如果有标题，就显示标题\n",
    "            plt.title(title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "728446d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAM8AAACeCAYAAACVU14NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8+yak3AAAACXBIWXMAAAsTAAALEwEAmpwYAAARVklEQVR4nO3de3BUZZrH8e/T3Umai7kQQgMhQCOIJCwlEBEdwUHUQWURS5lVKAddFUSnytlldsfd2YtjzW6tte44zo7l6K6uzkqtMBdGS9dFQXR1akS8jGK4JQGBhDtGCISEJP3sH+dNpskSkhzSnE7yfKpO5fR7bu9Lzi/v6dPNe0RVMcZ0XSjoChjTU1l4jPHJwmOMTxYeY3yy8Bjjk4XHGJ8sPGlCRI6LyJjuWldEvi4iVd1TOxARFZGx3bW/3iASdAX6IhH5AogBzUnFF6nq3s5sr6oDU1Ev0zUWnuD8saquDboSxj+7bEsTyZdFIvK8iDwpIq+JSK2IbBCRC9tZ9wYR2ezWqxaR77bZ73IROSgi+0TkrqTyLBF5TER2i8gBEfmZiPRLWv4Xbpu9IvKnqf8X6HksPOnrNuAHQB5QAfxDO+s9CyxV1QuAicBbScuGAjlAIXA38KSI5Lll/wRcBFwCjHXr/B2AiMwBvgtcC4wDrumuRvUmFp7g/EZEvnLTb86wfLWqfqCqTcAKvJP8TBqBYhHJVtUaVf24zbJHVLVRVf8bOA6MFxEBlgB/pqpfqmot8I94gQX4JvAfqvq5qp4AHj7HtvZKFp7gzFfVXDfNP8Py/UnzdUB7NwluAW4AdonIOyJyedKyIy58bfdTAPQHPmoJMPA/rhxgOLAnabtdnWxTn2I3DHo4Vd0I3CQiGcC3gVVAUQebHQZOAiWqWn2G5fva7GNkd9S1t7GepwcTkUwRWSQiOaraCBwDEh1tp6oJ4N+Ax0VkiNtXoYh8w62yCrhTRIpFpD/w9ylqQo9m4en57gC+EJFjwH3Aok5u9z28GxHvu23XAuMBVPV14Md4Nx8qOP0mhHHE/jOcMf5Yz2OMTykJj4jMEZFtIlIhIg+l4hjGBK3bL9tEJAxsx/uArQrYCNyuqpu79UDGBCwVPc80oEJVd6jqKeAl4KYUHMeYQKXic55CTv+ArQq4rO1KIrIE71NugKkpqIcx3UJV5UzlgX1IqqrPAM+A90XHoOphjF+puGyr5vRPp0e4MmN6lVSEZyMwTkTiIpKJ92XDV1JwHGMC1e2XbaraJCLfBtYAYeA5VS3r7uMYE7S0+IaBvecx6ay9Gwb2DQNjfLLwGOOThccYnyw8xvhk4THGJwuPMT5ZeIzxycJjjE8WHmN8svAY45OFxxifLDzG+GThMcYnC48xPll4jPHJwmOMTxYeY3yy8Bjjk4XHGJ86DI+IPOceCPt5UtkgEXlTRMrdzzxXLiLyEzdG9WciMiWVlTcmSJ3peZ4H5rQpewhYp6rjgHXuNcD1eA+AHYc3GuhT3VNNY9JPh+FR1f8FvmxTfBPwgpt/AZifVP5z9bwP5IrIsG6qqzFpxe97npiq7nPz+4GYmz/TONWFZ9qBiCwRkQ9F5EOfdTAmUOc86KGqqp9x12ysatPT+e15DrRcjrmfB125jVNt+gy/4XkFWOzmFwMvJ5V/y911mw4cTbq8M6Z3UdWzTsB/AfuARrz3MHcD+Xh32crxnqI8yK0rwJNAJbAJKO1o/247tcmmdJ3aO29trGpjOmBjVRvTzSw8xvhk4THGJwuPMT5ZeIzxycJjjE8WHmN8svAY45OFxxifLDzG+GThMcYnC48xPll4jPHJwmOMTxYeY3yy8Bjjk4XHGJ8sPMb4ZOExxqfOjFVdJCLrRWSziJSJyIOu3MarNn1aZ3qeJmC5qhYD04EHRKQYG6/a9HWdGRqqzTBRLwPXAtuAYa5sGLDNzT8N3J60fut6NvSUTT1xau+87dJ7HhEZDUwGNnCO41XbWNWmp+t0eERkIPAr4Duqeix5mXrdh3blwKr6jKqWqmppV7YzJl10KjwikoEXnBWq+mtXbONVmz6tM3fbBHgW2KKqP0paZONVmz6tw+F2ReRK4F28sacTrviv8d73rAJGAruAb6rqly5sP8V7mlwdcJeqnvV9jQ23a9JZe8Pt2ljVxnTAxqo2pptZeIzxycJjjE8WHmN8svAY45OFxxifzvlR8sa0JRImJ3sYzc2NJBLNNDc30NjUgCaaUVq+DJzaTydEhKKiImbOnImIUF9fT0NDwxl/NjU1tTudOHGi3WNYeNLMxInQ0AAHDkBtLQTxMdzAgTBhAuzeDUePQn1917YfMugibrnuX4lGcmlqrqe+sZb6U1/R2FRHw6njHD95kN9v/QUHD29vdx95WVmcbGqivrm5S8cOhUKMGzeOu+66i4ULF1JY6H0nue03ohOJROt8U1MTjY2NrT+T5xcsWNDusSw8aSQchuXLYexY76TduRO2boUdO6C8HKqqun4i+zFqFDzxBJw86YV4927YtMmrQ3U17N179lAPLSghp18RWZELWsuSP4xPaCN7D312WngECInQ7NYryctjz/Hj7Dp+vFN1jkQiTJo0iSVLlnDzzTdTUFCA92WXc9OvX7/2j3nOezfdLhr1poICKC6Ggwdh2zZYscI7gc+HUAiys70pHodJk7wwf/45PPec1zueiRBi+JBJZIRPP+mST2QhTGbGgNOWXxqLcf3IkTyycSMK/Hb//k5d2GVmZjJ9+nSWLVvGnDlzyMnJ6ZbQdIaFJ80cPpzNG28co6LCO1GrquCrr7xe4Pzpx5YtzWzadIqKCq/n27MHTpyApqazb5mVOZBY/gSEcLvrCMLwwZMoH/gWQ/LHs2P3byk7coQDdXWtgekoOAMGDGDWrFncf//9XHXVVfTr1++8haaFhccJhbwbj4lEooM1U0ckwpo1paxb91Yg73VaqI7m4Ydr2bWrqtPbZGVdwNCCizl+4gjZ/Yd1sLYQCoWZcOH1fO2PHuDND35IWfmr7Kqt7fA4ubm53HjjjSxdupRp06aRmZl53kPTos+HJysriyuuuIJ77rmHrKwsKioqKC8vp7Kykurqag4dOkRdXR2nTp06L/VRlUCD4xG68j49HMpg1rTlFMVK+WTbS0Qzczs8oUXCfLZlNYNzxnLdZX+LaoLNFa/R3heVhwwZwi233MK9997LxIkTycjI6EqDUqLPhmfAgAFcc801LFu2jBkzZrR2+y13YJqbmzlx4gQ1NTXs2bOHL774gu3bt1NRUcHOnTvZt28fNTU1rbc6+7Jx8auZMPoG3tjwCI2N9YQk3BqC9kLU2HSS+lPHWPu7RwG4Ztr3OVa3jz3VH7Wu03K7edGiRdx5551ceOGFhMPtXw6eb30uPHl5ecydO5f77ruPqVOn/r9uX0QQEUKhELm5ueTm5hKPx5k5c2brLc7GxkZqa2s5dOgQu3fvprKykvLycsrKytiwYQO1nbj86C2yBw5l5pQH2bZrDVsr1xDLv5gtu18jK+MCcqI5ZGYMJCFZhEOZZESiRML9CEsGdfVHADjVeJy3N/4LtQ2VhDOOMXnyZIYPH87o0aMpKSlh7ty5FBYWtl5Wp5M+E55YLMaCBQu45557KCkpIRwOd/laWUQIh8OEw2Gi0SgFBQUUFxe3/pVtbGzk008/5dlnn2X16tUcPHiwgz32fAX54zl6oop3P/4pzc2N7D24iVfWfY9wKMLAjCwkFOYUEI0OID8vRiw2guGFIyie0p8rr/kb4mPijBo1imHDhjF48KMMGDCAaDTaGpag3s90Rq8Oj4gwcuRI7rjjDhYvXkw8Hk9Jt9/yC87MzOTSSy9lypQpLF++nBdffJEVK1awc+fOQG9EpNKOXe9Svf8T6hv+MCZMdvZAJk+eTElJCfF4nNGjR1NUVMSQIUPIzc2lf//+hMNhQqFQWoejQ10dty0VE908zlYoFNIJEyboY489plVVVZpIJDQIiURC9+/fr0899ZSWlpZqRkbGWesdiUR09uzZgY9TVlxcrCNGjOjSNrm5uTp79mx94okndMuWLdrQ0KCJRKJ16qmmTp2q2s5526t6noyMDC655BKWLFnC/Pnzyc/PD/Qvm4gQi8VYunQpCxcuZO3atTz99NO899571NXVBVav7iAi5OTkMHXqVObNm8e1117LmDFjAr11fN61lyr9Q68QBT4APgXKgB+48jjeICAVwEog05VnudcVbvnoThzD91/JjIwMzc7O1lmzZumqVav06NGjaf2X7uTJk/rOO+/ookWLNC8vr8f1PMk9zObNm7W+vj6t/73P1bn2PA3A1ap63I3f9p6IvA78OfC4qr4kIj8D7sYbl/puoEZVx4rIbcCjwJ904jhnFA6HiUQiZGdnk5eXR2FhIUVFRcTjccaMGUNRURHDhw9n5MiRRKPRtP+rF41GmTFjBldccQVbt27l+eefZ+XKlVRXp+fQdtbDtK9Lo+eISH/gPWAZ8BowVFWbRORy4GFV/YaIrHHzvxORCN5QvAV6lgOJiGZnZ5OdnU0sFqOwsJB4PE487t2JKSoqYvDgweTl5RGNRolEIml569KPRCJBVVUVK1eu5MUXXyQ/P5/169cHWqfx48cTDocZOnQo8+bN47rrruuzgSktLeXDDz/0P/SUiISBj4CxwJPAPwPvq+pYt7wIeF1VJ4rI58AcVa1yyyqBy1T1cJt9LsF7igKxWGzq22+/zeDBg+nfv/9pPUhf+WWpKjU1NZSVlXG8k98kTpVIJMKoUaMYNWpUnwxMsrOFp1M3DFS1GbhERHKB1cDF51opVX0GeMZVUC+++Jx32aOJCIMGDWLGjBlBV8V0UpeufVT1K2A9cDmQ6y7L4PTxqFvHqnbLc4Aj3VFZY9JJZ8aqLnA9DiLSD+/ZPFvwQnSrW20xp49VvdjN3wq8dbb3O8b0VJ25bBsGvODe94SAVar6qohsBl4SkR8Cn+ANBo/7+Z8iUgF8CdyWgnobE7gOw6Oqn+E90Kpt+Q5g2hnK64H2/+O3Mb1E77jfa0wALDzG+GThMcYnC48xPll4jPHJwmOMTxYeY3yy8Bjjk4XHGJ8sPMb4ZOExxicLjzE+WXiM8cnCY4xPFh5jfLLwGOOThccYnyw8xvhk4THGJwuPMT5ZeIzxycJjjE9dGug9ZZUQqQW2BV2PFBoMHO5wrZ6rN7dvlKoWnGlBujzcapuqlgZdiVQRkQ+tfb2PXbYZ45OFxxif0iU8zwRdgRSz9vVCaXHDwJieKF16HmN6HAuPMT4FHh4RmSMi20SkQkQeCro+XSUiRSKyXkQ2i0iZiDzoygeJyJsiUu5+5rlyEZGfuPZ+JiJTgm1B54hIWEQ+EZFX3eu4iGxw7VgpIpmuPMu9rnDLRwda8RQKNDzugVlPAtcDxcDtIlIcZJ18aAKWq2oxMB14wLXhIWCdqo4D1rnX4LV1nJuWAE+d/yr78iDeEwFbPAo87h7qXAPc7crvBmpc+eNuvV4p6J5nGlChqjtU9RTwEnBTwHXqElXdp6ofu/lavBOsEK8dL7jVXgDmu/mbgJ+r5328Z7sOO7+17hoRGQHcCPy7ey3A1cAv3Spt29fS7l8Cs6WXPk476PAUAnuSXle5sh7JXaJMBjYAMVXd5xbtB2Juvie2+cfAXwIJ9zof+EpVm9zr5Da0ts8tP+rW73WCDk+vISIDgV8B31HVY8nL3AONe+RnAiIyFzioqh8FXZd0E/R321ofO+8kP5K+xxCRDLzgrFDVX7viAyIyTFX3ucuyg668p7X5a8A8EbkBiALZwBN4l5sR17skt6GlfVUiEgFygCPnv9qpF3TPsxEY5+7cZOI9OfuVgOvUJe56/llgi6r+KGnRK8BiN78YeDmp/Fvurtt04GjS5V3aUdW/UtURqjoa7/fzlqouAtYDt7rV2ravpd23uvV7ZK/bIVUNdAJuALYDlcD3g66Pj/pfiXdJ9hnwezfdgHedvw4oB9YCg9z6gneHsRLYBJQG3YYutPXrwKtufgzwAVAB/ALIcuVR97rCLR8TdL1TNdnXc4zxKejLNmN6LAuPMT5ZeIzxycJjjE8WHmN8svAY45OFxxif/g9lIKJify7WuQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 216x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 导入gym库\n",
    "import gym\n",
    "\n",
    "# 创建LunarLanderContinuous环境，指定渲染模式为rgb_array，如果是在IDE中可以改为'human'\n",
    "env = gym.make('LunarLanderContinuous-v2', render_mode='rgb_array', continuous='False')\n",
    "# 重置环境\n",
    "env.reset()\n",
    "# 创建GymHelper\n",
    "gym_helper = GymHelper(env)\n",
    "\n",
    "# 循环N次\n",
    "for i in range(100):\n",
    "    gym_helper.render(title = str(i)) # 渲染环境\n",
    "    action = env.action_space.sample() # 从动作空间中随机选取一个动作\n",
    "    observation, reward, terminated, truncated, info = env.step(action) # 执行动作\n",
    "    if terminated or truncated: # 如果游戏结束，则结束循环\n",
    "        break\n",
    "\n",
    "# 游戏结束\n",
    "gym_helper.render(title = \"Finished\")\n",
    "# 关闭环境\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6bde6585",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.4609097 , -0.13413402], dtype=float32)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "action = env.action_space.sample() \n",
    "action"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4f75653",
   "metadata": {},
   "source": [
    "### DDPG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1a7d045a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "# 演员网络（策略网络）\n",
    "class ActorNetwork(nn.Module):\n",
    "    def __init__(self, state_dim, action_dim):\n",
    "        super(ActorNetwork, self).__init__()\n",
    "        \n",
    "        self.fc = nn.Sequential(\n",
    "            nn.Linear(state_dim, 512),\n",
    "            nn.LayerNorm(512),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(512, 256),\n",
    "            nn.LayerNorm(256),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(256, action_dim)\n",
    "        )\n",
    "        self.optimizer = optim.Adam(self.parameters(), lr=0.0005)  # 使用Adam优化器\n",
    "        self.apply(weight_init)  # 初始化模型权重\n",
    "\n",
    "    def forward(self, state):\n",
    "        # 前向传播过程\n",
    "        action = torch.tanh(self.fc(state))  # 输出动作，通过tanh激活函数归一化\n",
    "        return action\n",
    "\n",
    "# 评论家网络（价值网络）\n",
    "class CriticNetwork(nn.Module):\n",
    "    def __init__(self, state_dim, action_dim):\n",
    "        super(CriticNetwork, self).__init__()\n",
    "        \n",
    "        self.fc = nn.Sequential(\n",
    "            nn.Linear(state_dim, 512),\n",
    "            nn.LayerNorm(512),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(512, 256),\n",
    "            nn.LayerNorm(256)\n",
    "        )\n",
    "        self.fc2 = nn.Linear(action_dim, 256)  # 输入动作的全连接层\n",
    "        self.q = nn.Linear(256, 1)  # 输出Q值的线性层\n",
    "\n",
    "        self.optimizer = optim.Adam(self.parameters(), lr=0.0005, weight_decay=0.001)  # 使用Adam优化器，并设置权重衰减\n",
    "        self.apply(weight_init)  # 初始化模型权重\n",
    "\n",
    "    def forward(self, state, action):\n",
    "        # 前向传播过程\n",
    "        x_s = self.fc(state)  # 状态通过第一个全连接层后接ReLU激活函数和批归一化层\n",
    "        x_a = self.fc2(action)  # 动作通过全连接层\n",
    "        x = torch.relu(x_s + x_a)  # 状态特征和动作特征相加后接ReLU激活函数\n",
    "        q = self.q(x)  \n",
    "        return q # 输出Q值\n",
    "\n",
    "\n",
    "def weight_init(m):\n",
    "    # 初始化模型权重的函数\n",
    "    if isinstance(m, nn.Linear):\n",
    "        nn.init.xavier_normal_(m.weight)\n",
    "        if m.bias is not None:\n",
    "            nn.init.constant_(m.bias, 0.0)\n",
    "    elif isinstance(m, nn.BatchNorm1d):\n",
    "        nn.init.constant_(m.weight, 1.0)\n",
    "        nn.init.constant_(m.bias, 0.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "041657ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 经验回放缓冲区\n",
    "class ReplayBuffer:\n",
    "    def __init__(self, max_size, state_dim, action_dim, batch_size):\n",
    "        # 初始化回放缓冲区的大小、状态维度、动作维度和批次大小\n",
    "        self.mem_size = max_size\n",
    "        self.batch_size = batch_size\n",
    "        self.mem_cnt = 0\n",
    "\n",
    "        # 创建用于存储经验的数组\n",
    "        self.state_memory = np.zeros((self.mem_size, state_dim))  # 存储状态\n",
    "        self.action_memory = np.zeros((self.mem_size, action_dim))  # 存储动作\n",
    "        self.reward_memory = np.zeros((self.mem_size, ))  # 存储奖励\n",
    "        self.next_state_memory = np.zeros((self.mem_size, state_dim))  # 存储下一个状态\n",
    "        self.terminal_memory = np.zeros((self.mem_size, ), dtype=bool)  # 存储终止标志\n",
    "\n",
    "    def store_transition(self, state, action, reward, next_state, done):\n",
    "        # 存储经验转换\n",
    "        mem_idx = self.mem_cnt % self.mem_size\n",
    "\n",
    "        self.state_memory[mem_idx] = state\n",
    "        self.action_memory[mem_idx] = action\n",
    "        self.reward_memory[mem_idx] = reward\n",
    "        self.next_state_memory[mem_idx] = next_state\n",
    "        self.terminal_memory[mem_idx] = done\n",
    "\n",
    "        self.mem_cnt += 1\n",
    "\n",
    "    def sample_buffer(self):\n",
    "        # 从缓冲区中随机采样一个批次的经验转换\n",
    "        mem_len = min(self.mem_size, self.mem_cnt)\n",
    "        batch = np.random.choice(mem_len, self.batch_size, replace=False)\n",
    "\n",
    "        states = self.state_memory[batch]\n",
    "        actions = self.action_memory[batch]\n",
    "        rewards = self.reward_memory[batch]\n",
    "        next_states = self.next_state_memory[batch]\n",
    "        terminals = self.terminal_memory[batch]\n",
    "\n",
    "        return states, actions, rewards, next_states, terminals\n",
    "\n",
    "    def ready(self):\n",
    "        # 判断缓冲区是否已准备好进行采样\n",
    "        return self.mem_cnt >= self.batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f9019251",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class DDPG:\n",
    "    def __init__(self, env, gamma=0.99, tau=0.005, action_noise=0.1, max_size=1000000, batch_size=256):\n",
    "        state_dim=env.observation_space.shape[0]\n",
    "        action_dim=env.action_space.shape[0]\n",
    "        batch_size=256\n",
    "        # 初始化DDPG算法的超参数和网络模型\n",
    "        self.gamma = gamma  # 折扣因子\n",
    "        self.tau = tau  # 软更新系数\n",
    "        self.action_noise = action_noise  # 动作噪声幅度\n",
    "        \n",
    "        # 判断可用的设备是 CPU 还是 GPU\n",
    "        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "        \n",
    "        self.actor = ActorNetwork(state_dim=env.observation_space.shape[0],\n",
    "                                  action_dim=env.action_space.shape[0]).to(self.device) # 创建演员网络\n",
    "        self.target_actor = ActorNetwork(state_dim=env.observation_space.shape[0],\n",
    "                                  action_dim=env.action_space.shape[0]).to(self.device) # 创建目标演员网络\n",
    "        self.critic = CriticNetwork(state_dim=env.observation_space.shape[0],\n",
    "                                  action_dim=env.action_space.shape[0]).to(self.device)  # 创建评论家网络\n",
    "        self.target_critic = CriticNetwork(state_dim=env.observation_space.shape[0],\n",
    "                                  action_dim=env.action_space.shape[0]).to(self.device)  # 创建目标评论家网络\n",
    "\n",
    "        self.memory = ReplayBuffer(max_size=max_size, state_dim=env.observation_space.shape[0],\n",
    "                                  action_dim=env.action_space.shape[0], batch_size=batch_size)  # 创建回放缓冲区\n",
    "        \n",
    "        self.update_network_parameters(tau=1.0)  # 初始化目标网络参数与主网络参数相同\n",
    "\n",
    "    def update_network_parameters(self, tau=None):\n",
    "        # 更新目标网络参数\n",
    "        if tau is None:\n",
    "            tau = self.tau\n",
    "\n",
    "        for actor_params, target_actor_params in zip(self.actor.parameters(),\n",
    "                                                     self.target_actor.parameters()):\n",
    "            target_actor_params.data.copy_(tau * actor_params + (1 - tau) * target_actor_params)\n",
    "\n",
    "        for critic_params, target_critic_params in zip(self.critic.parameters(),\n",
    "                                                       self.target_critic.parameters()):\n",
    "            target_critic_params.data.copy_(tau * critic_params + (1 - tau) * target_critic_params)\n",
    "\n",
    "    def remember(self, state, action, reward, next_state, done):\n",
    "        # 存储经验转换\n",
    "        self.memory.store_transition(state, action, reward, next_state, done)\n",
    "\n",
    "    def choose_action(self, observation, train=True):\n",
    "        # 根据当前状态选择动作\n",
    "        self.actor.eval()\n",
    "        state = torch.FloatTensor(observation).to(self.device)\n",
    "        action = self.actor.forward(state).squeeze()\n",
    "\n",
    "        if train:\n",
    "            noise = torch.tensor(np.random.normal(loc=0.0, scale=self.action_noise),\n",
    "                             dtype=torch.float).to(self.device)\n",
    "            action = torch.clamp(action + noise, -1, 1)\n",
    "            return action.cpu().detach().numpy()\n",
    "        self.actor.train()\n",
    "        return action.cpu().detach().numpy()\n",
    "\n",
    "    def learn(self):\n",
    "        # 使用经验回放进行训练\n",
    "        if not self.memory.ready():\n",
    "            return\n",
    "        \n",
    "        # 从回放缓冲区中获取一批样本\n",
    "        states, actions, rewards, next_states, dones = self.memory.sample_buffer()\n",
    "        states = torch.FloatTensor(np.array(states)).to(self.device)\n",
    "        actions = torch.FloatTensor(np.array(actions)).to(self.device)\n",
    "        rewards = torch.FloatTensor(np.array(rewards)).to(self.device)\n",
    "        next_states = torch.FloatTensor(np.array(next_states)).to(self.device)\n",
    "        dones = torch.LongTensor(np.array(dones)).to(self.device)\n",
    "        \n",
    "        # 使用目标演员网络生成下一个状态对应的下一个动作，并计算目标评论家网络对应的 Q 值。\n",
    "        with torch.no_grad():\n",
    "            next_actions = self.target_actor.forward(next_states)\n",
    "            q_ = self.target_critic.forward(next_states, next_actions).view(-1)\n",
    "            q_[dones] = 0.0\n",
    "            target = rewards + self.gamma * q_\n",
    "        \n",
    "        # 使用当前评论家网络计算当前状态和动作对应的 Q 值\n",
    "        q = self.critic.forward(states, actions).view(-1)\n",
    "\n",
    "        # 计算评论家网络的损失，并根据损失进行反向传播和优化\n",
    "        critic_loss = F.mse_loss(q, target)\n",
    "        \n",
    "        self.critic.optimizer.zero_grad()\n",
    "        critic_loss.backward()\n",
    "        self.critic.optimizer.step()\n",
    "        \n",
    "        # 生成当前状态对应的新动作，并计算演员网络的损失\n",
    "        new_actions = self.actor.forward(states)\n",
    "        actor_loss = -torch.mean(self.critic(states, new_actions))\n",
    "        \n",
    "        # 根据演员网络的损失进行反向传播和优化\n",
    "        self.actor.optimizer.zero_grad()\n",
    "        actor_loss.backward()\n",
    "        self.actor.optimizer.step()\n",
    "        \n",
    "        # 更新目标网络的参数。\n",
    "        self.update_network_parameters()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "53e1d859",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 0: -784.6120014944543           \n",
      "Episode 40: -747.1179881203424                     \n",
      "Episode 80: -615.8579268248366                     \n",
      "Episode 120: -242.75193766510196                    \n",
      "Episode 160: -103.31275144136711                    \n",
      "Episode 200: -269.8050393933729                     \n",
      "Episode 240: -145.80199143510367                    \n",
      "Episode 280: -369.9257554573615                     \n",
      "Episode 320: -366.54690172499164                    \n",
      "Episode 360: -341.05775547691803                     \n",
      "Episode 400: -281.8020043039842                        \n",
      "Episode 440: -119.58064891601806                       \n",
      "Episode 480: -149.43468297292503                       \n",
      "Episode 520: -243.9350084282831                        \n",
      "Episode 560: 107.38898575497666                        \n",
      "Episode 600: -16.968212548217707                       \n",
      "Episode 640: 198.59606799215874                        \n",
      "Episode 680: -97.96116336907137                       \n",
      "Episode 720: -131.5897324160774                       \n",
      "Episode 760: 101.84653666315026                       \n",
      "Episode 800: 4.101861778524837                        \n",
      "Episode 840: -198.07537561919895                      \n",
      "Episode 880: -124.20416247737687                      \n",
      "Episode 920: -149.37193343992823                      \n",
      "Episode 960: -75.291632700399                         \n",
      "Episode 1000: 257.6968375049101                        \n",
      "Episode 1040: 168.2279980060572                        \n",
      "Episode 1080: 227.09376062872042                       \n",
      "Episode 1120: 231.0825152687513                        \n",
      "Episode 1160: -100.24106118224299                      \n",
      "Episode 1200: 213.78699192963052                       \n",
      "Episode 1240: 233.4880047983544                        \n",
      "Episode 1280: 101.60903512860067                       \n",
      "Episode 1320: 250.54307022824608                       \n",
      "Episode 1360: 203.34565965210106                       \n",
      "Episode 1400: -64.3884021789394                        \n",
      "Episode 1440: 260.8365294758823                        \n",
      "Episode 1480: 182.584071318447                       \n",
      "100%|██████████| 1500/1500 [9:08:38<00:00, 21.95s/it]\n"
     ]
    }
   ],
   "source": [
    "# 导入必要的库\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import numpy as np\n",
    "import sys\n",
    "import time\n",
    "import random\n",
    "import collections\n",
    "from tqdm import * # 用于显示进度条\n",
    "\n",
    "# 定义超参数\n",
    "max_episodes = 1500 # 训练episode数\n",
    "max_steps = 1000 # 每个回合的最大步数\n",
    "\n",
    "# 创建DDPG对象\n",
    "agent = DDPG(env)\n",
    "# 定义保存每个回合奖励的列表\n",
    "episode_rewards = []\n",
    "\n",
    "# 开始循环，tqdm用于显示进度条并评估任务时间开销\n",
    "for episode in tqdm(range(max_episodes), file=sys.stdout):\n",
    "    # 重置环境并获取初始状态\n",
    "    state, _ = env.reset()\n",
    "    # 当前回合的奖励\n",
    "    episode_reward = 0\n",
    "\n",
    "    # 循环进行每一步操作\n",
    "    for step in range(max_steps):\n",
    "        # 根据当前状态选择动作\n",
    "        action = agent.choose_action(state)\n",
    "        \n",
    "        # 执行动作，获取新的信息\n",
    "        next_state, reward, terminated, truncated, info = env.step(action)\n",
    "        done = terminated or truncated\n",
    "        \n",
    "        # 将五元组加到经验回放缓冲区\n",
    "        agent.remember(state, action, reward, next_state, done)\n",
    "        agent.learn()\n",
    "        episode_reward += reward\n",
    "        \n",
    "        # 更新当前状态\n",
    "        state = next_state\n",
    "        if done:\n",
    "            break\n",
    "    # 记录当前回合奖励值\n",
    "    episode_rewards.append(episode_reward)\n",
    "\n",
    "    # 打印中间值\n",
    "    if episode % 40 == 0:\n",
    "        tqdm.write(\"Episode \" + str(episode) + \": \" + str(episode_reward))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a005db53",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEICAYAAACwDehOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8+yak3AAAACXBIWXMAAAsTAAALEwEAmpwYAAA7fUlEQVR4nO2dd5wV1dnHv892loWl97J0pNmQoogFUewaNRE1lhiNib4p+ibRGI0xakxMNfGNPZpijF1jAQWNHRQEpMNSpFfpsMuW8/5xZ2bn3jtzd26/7D7fz2fh3jNnZp577tzzO+d5ThFjDIqiKIoCkJdtAxRFUZTcQUVBURRFcVBRUBRFURxUFBRFURQHFQVFURTFQUVBURRFcVBRUJQcRkSuFJEPsm2H0nxQUVAURVEcVBQUxQMRKWgO91SUSFQUFMVCRFaLyI9F5HNgn4iME5GPRGSniMwTkROtfCeJyHzXeW+JyKeu9++LyHnW65tFZIWI7BGRRSJyvivflSLyoYj8XkS2A3eISHsReUVEdovIJ0C/zHx6RQmhLRNFCWcycCZQD3wOfB2YAkwAnheRwcAMYICIdAB2ASOAWhFpBdQCI4H3reutAI4HNgEXAf8Qkf7GmI3W8dHA00BnoBD4K1AFdAX6AFOBVen8wIriRnsKihLO/caYtcBlwOvGmNeNMfXGmLeAWcAZxpgDwKfAeOBoYB7wIXAcMAZYbozZDmCMedYYs8G6xr+B5cAo1/02GGP+ZIypBQ4CFwC3G2P2GWMWAE9m5FMrioX2FBQlnLXW/72Bi0TkbNexQuAd6/W7wInAOuv1DuAEoNp6D4CIXA7cCFRYSWVAB4/7AXQk9Jt0p32R8CdRlARQUVCUcOxlg9cCfzfGXOOT713gt8Aa4F5CovAIIVF4AEBEeltpE4CPjTF1IjIXEI/7AWwl5H7qCSyx0nol+XkUJS7UfaQo3vwDOFtEThORfBEpEZETRaSHdfwjYBAhV9AnxpiFhHoXo4H3rDwtCVX6WwFE5CpgmN8NjTF1wAuEAs6lIjIEuCINn01RfFFRUBQPrLjCucBPCFXqa4EfYv1mjDH7gM+AhcaYg9ZpHwNfGGO2WHkWEepNfAxsBoYTij3E4gZCLqZNwBOEAs+KkjFEN9lRFEVRbLSnoCiKojioKCiKoigOKgqKoiiKg4qCoiiK4nDIz1Po0KGDqaioyLYZiqIohxSzZ8/eZozpGJl+yItCRUUFs2bNyrYZiqIohxQi4jlbXt1HiqIoioOKgqIoiuKgoqAoiqI4qCgoiqIoDioKiqIoioOKgqIoiuKgoqAoiqI4qCgoitIkMMbw0pz1bNtbDcDB2np0Fej4UVFQFKVJsGD9br7/77n89s2l7DpQw7CfTeXR91cFOtcYw9tLNlNXryKioqA0WzbtqmL2F196HttTVUNVTZ3z/vnZ66i4+TV2HajJlHnU1Rsqt+xN6hpvL9nMgYN1jWdMkkfeW+lblm527j/IYx+simrBf1i5jVXb9gGw9sv9bLda+3/9cBUL1u+iujb0GbbtreY3U5fyqymh3Upr6ur51ydrqKs37KkOfTfvL9/Gjf+ey8G6eu5+fXEg+/+7bCvfeGIWD7xTGTPf/oO13PXqIiq37GHgT99g9hc72LDzgG/+fdW1LFi/i827qwLZ4cXTn6xhfYx7pJpDfpkLRUmUib97lz3Vtay+98yoY8PveJNBnVsx9QfjAXjw3RVASEjKWxSm1a66esNjH6xk+76DPPTuSt76wXgGdG4V93XmrNnBN56YxZXHVnDHOUPZvreaqtp6urdp4XvOnqoaNu2q8rzfP2d+wUmDOtEt4vxXP9/gVL5PXTOal+asp3+nMu55fQl3nD2EB99dybs/OpHignxuemYe05dsYWTvthzes41zjUsfnQnA6nvP5PhfvwPAO/97Ij//zyIAhnRtzV8uO4oT7vuvc85Rvdry2zeXsmTTHmrrDbe9tACAdTsOsG5HQyW6ZU8Vo+6ezgOXHMWZI7p6fu791SHRWbhhl5O29sv9vDJvAxcd3YO7XlvMvRcM55H3VvHoB6t4bf5GDtbWc8FfPnLstpm/bhddykvo2KqYoT+bCkB+nrDinjNYv/MA1TV19O1Yxva91Rx91zQAPr7lZNq0KKIwXyjIb2irr9y6l5tfmE9JYR6L75zElAWbAOjYqpg5a3Zyzfi+np8nGVQUlGbLnuramMeXbt7jvD5YVw9AYb4Evv7Lc9fTtrSI8QOj1hxj7Zf7OVhXT7+OZVHHXp+/kXteX+K8X7fjAFMWbGL19v3ceuZhtGtZFOj+a77cD+D42EfePQ1jQhXY/HW7GNa9NSLhn+fih2ewcMPusEquqqaO/3unkvvfrqRlUT4L75wUds4NT81xXl/yyMywY3dYlfqnq3aw5sv9TF+yBYA8674fLN/Gc7PXetp/zp8/cF4v2rg7TBAArvlbw5pnM1du9ykFGHX3dACe+GiVIwr3T1/O795axpJfTKK4II+yklBVuPtALVU1ddzw1BymLd4MwH1TlwJwWNfWTlkWRDwHFTe/xvPfPpaje7flbJfdNrZb6rh73wbg5MGdyM9ruMbE373HXut5/OFpg7hv6lJW3HMGJ//2XQCqaur52sMz+GRVeG9MRUFRIpi+eDMtiwsY07d9WPrqbftoW1pEeWnyrfp1O/bzxfZQBVuYH9vjuruqhrKiAmrq6/ne03PDjq365RlOJWy3ht2V75MfraZreUmUX3v6ks38Y8YaAJ7/bB3/uWEcw3uUc/Pzn/P0p2v52dlDuOq4PmHnrN95wLl/a6tnY3ts3lq0mWv+Nov7LhzBRSN7hp23cMNuIOSWsT/rxQ/PYO7anQDsO1jHzJXbGdSlFW1Kg4kTwGWPzaRH24Yehghc8sgMPloRXpl/WLnNeb2nKrZou3n1842N5vlszU4mPzyDC4/uwV8/DMUaBt82hTOHd2Vsv9DzU1NXz0crtjmC4MZ2WQGs/TLanfPH6ct58qpjfO+/p6rB9fj2ki1hPaW9rgaKLUI1VkPEJlIQAOrrDXl5wRsqQVBRUA5prn4y1Fp89rqxHNWrrdP6OvE3/6V3+1Le/eFJGGO445WFlBTl89C7K1nw89MoKw726FfV1HH1Ew0t0om/f5clvzjdM++Nz8zlhc/W860T+vLesm1Rxz9ftyusIoBQj+HdZVu5bExvfvbKQgD+culRYXlsQbB5+tM1FBVU8PSnoRb2z/+zKEoU3C3np2au4Z7zhzvvbR/4vHU7uWhkT6pr6/jhs58zZ+0OJ8/+6jqufOJjTh3SxREEm689PIPDe5Qzb90u4sHt0jnrT9GtaWhwI6WDunrDxyu38/HK7XRsVeykvzZ/I6/ND4lKTV09LQoTqxbfW7aV3765zPf48DveDHs/L6JcIxl825RG71ldW0+LovxA9gVFRUFpElz04McAnD6si+NesVv3W/dW8+THDasEV27ZyxERlbMfkT/Mqpp6n5zwwmfrAXjo3ZWex+2WtzvIOvmRGazbcYDj+ndw0ooLY/dG/jlzDf+cGS4UM1dup0VRPk98tJqvj+nNS3M3+J7f0hLEfZYffcqCTbwyLzz/jv0HmbNmJ3PW7PS8RryCkGvU1nl/j/PW7eKSR2ckfN0/NxKoTjXVtXUqCooSizesQJzNkx+t5ozh4cHFrXuqk7rHss17GJhA4LfEquztFj7Ajn0HATjpN/910p74yHOZ+5h87eGGiswWJz/sXpLtzvAaym+7MJoqO/b7jyI7lKY2xGqkJIoOSVWaND97ZSHPf7YuLM0doEyE9TsSGx5oB1fdfvN9HsNF31u2NTHDYnDVXz9xXn+8InT/JZv2+GV33Cm5yA0n9c+2CTmDe9h0qlBRUJo8jc0tmBLRu2iM95dHxwuCYAiNBAoSFE017yxtEBrblbZux4G0VCrpprFgfyrp0rokY/dKhPwUB5lBRUFpBtQ3Mkv1un/MjmuC1+MfBpslG8meqhpGWuPSc4XBt03hD9P8g6OJcNtZQ3yPHVPRNunrRw4HDcrlY3vHfY57xFCuMbhLK3q2K035dVUUlEMWv2BhVL4ASxe4g4sbd6Vn9qjXkMJcYLUVkE8VJwzsyMp7zvA89sjlI5O+vrt1fHiPcoZ0be2Z76sje4S9P7ZfB898sfBy79l0KAs+JNfND08bxCmHdQ6c/48XH+GZnqg4NoaKgnLIcv1TnwXKF2Q9mzlrdlJfb7j7tUWM/eXbyZrWrMkTfMfOC8lXZAWua/9p8lHU1ns3Du44Z2hU2vUn9Uv6/jaRE//8iOw5XT2uD49e0bg4fnfCAObcNpFjKtp5Hs8PeP94UVFQDlmmLoyeYOSF10qZSz2CrIs27uaRgAuo2WvxxEPQSuRQolVJ9ADGvBifU1JQ49g9hcmjetGrfSm1dd6iX1pUwNiISY3xtNBjccXY3lw9rk/jGYGigvAPHfQxaFmUT9uWRf5xAxUFRUmMOg9ROO0P70WlxarMIhn008YnFjUHzj+ye1RaTFFIwT3tnoIt9jUePQV7pnikKQV5qanyfn7uMEoDzg+I/Mzx9pb8yjMNMebQddNzWUXJHYIuhxzPukaJ0PT6CVBSGF0xxtLWeITXj3yrYq+3RMGvpxBNaicgnBchiI9f6e0SivzM8RaBX08hXc+TioLS5AkqCsnUV9MWNe7KaoLeI0os14h75dVYa/GkogzsnoI9zqAmhihE3i+V30HrkkJ+fcEI572f4EUmxxJG9/pQNn6xg1QIrOd103JVRckhAg5SCpzPi28GmBDXBDWBEsuF4o7bxAqA+lVkvdsHH1qZF+E+shezyzZB3TyxnoPvTRjQkM/KmO/Tg031QnjOddNyVUVJEXuqaliyaXdS16jzGZ0SyZ/eXp7UfZojJQXR7qNE6qqCOE6y567Z7qP7LhzBtBtP8Mwb6b+3K9p01Kf+PYXg7iOva/iJrLqPlGbJZY99wqQ/vJ/UNYK6nNM90zhdLbts4h1TiL+nEE8A2L6G7RUsKcynf6eGfSl+dnbDEFA/U1LlejGuOIXfJaMCzTHu7RU/8CsadR8pzZLGlhcOQmMzmjNF05OEhkX+3MTSPr9j8UzEEkcUvL/XyGXE3Qzs3IozhnfhjxcfGXXs2iQ3rEmFAHll9espfBxjY6Fk0FVSlSaPbsaePrx6CjGHpPr2FIJXnHZWP1Hwu58xoXWT/u/So1lt7QcdljewBe5zGs4KGmiOhVdPoSCDaz2B9hSUQwSvCWhB8ZqnkAritakpilOLOEXBfx5WPKJgjz5KvDy9bExkcqHbfeQfaI7/s2UTFQXlkCCZ+jRd7qN4tWbF1ujW6aFOcUF0FRJr1rJfxRtPuCUyphALv8t6+emD2vCLc6OXz4h1fjz1vIqCogQkiKvAj3T1FOK16e8z4t88J9fxCp4nUrHFc05+xJDURPAM6Aa04dj+3gvr+QlePD0Qt12pWCcqEXJOFERkkogsFZFKEbk52/bkIos37g7b6Ls5kIwobNpVlUJLGuh/6xsYY/jjtOY7lNWrIk1kkFV8LpbQ/4F6CuL9OpnF5Hx7H9aBtqWFYSu0Bi2P04d1CctrUjwDOyg5JQoikg88AJwODAEmi4j/4uzNkNq6ek7/4/tcm+TuYbnE9MWbuejBjzhYGz6f4OMVDaMr3Jqwats+fvtm8O0iY+0wliwLN+zm9ynej+BQwq7ETFha/BVuIi6WIA2Fu88f7rx2Z09mcUJ/F1hDer7LP+XX4r/quIqwVVsfuOSouIYtT/3++MB54yHXRh+NAiqNMSsBRORp4FxgUVatyiHsvQFmrd6RZUtSx9VPhgRu5/6DGKBNaSGrt+1n8iMNexy4K4Br/zaL5Vv2ZtpMTzK5C1gu4lU/2pXjrWccxrLNe3h29rroTAGu01jeID2F7m1acMFRPXj+s3Vh31UyO5b59xTii5f87OxQbOKBd1aE8uVJfIsydol/n/Ag5JoodAfWut6vA0ZnyZacxKkbsx+PSjk19Ybj7n2bYyra8mmE6LkrgBprPYpWJQXsqcquG80r0Nqc8Go125XgNda4/yHdWvPz/6SuXef0FAIOILj97CH06VDKyYM7OWle7qPG3DUti/LZd7Au7qGnccUUXHk1phAHInKtiMwSkVlbt6Z+k/NUs3zznsC7hDWGHTQ9lCbHbt5dFWi7y617qgGiBAHgoXdXsGV3KDZgt/IO1tYzzifol2oyvKT9IYN3TCE8LdZkMpuAK5EADd9/0DhTeYtCbjh5QJhrpsxjH4igg9TinaQWn2sseN50kWuisB7o6Xrfw0oLwxjzsDFmpDFmZMeOHTNmXCJ8sX0fE3//Hr+asiQsvb7eJDR6wh6bnQtD14Iy+p7pXPbYzEbznffAh77H/vR2Jb+x4gi2G6C6tj7KDXBEzzaJGxoDP3dDE5x6EBdepZLIoxlZwcfaq6DBfZTc6KPJo3rFdU5jLX47jGBc/0J8nfpc2Igp10ThU2CAiPQRkSLgYuCVLNuUED95cT4VN7/mtH4/W7PTOXawtp6+P3md374Zf4DSFoXsPzrxMfuL5GMgfTqE1rdxL4kQWVfHqiae/MYo7vQZY94Yfj/WZCqmpkCqJoHFM2zYb57CQ18/mh+eNijwdTpG7LFc3qIwZn67Eefucbg/ayoaajmgCbklCsaYWuAGYCqwGHjGGLMwu1YlxlMz1wAND677u66ytnL864fBtn50k8s9hZfnruejym1pu77tv3eP7Bjt2m7xnvOHx5xRNqZvOy4fW5HQvf269bmyrlK2sB9DY0Ll77UfQBDiKUf72Y/saZ82tAvXn9Q/+E0jfkMTBndigivu4Hua6/W5R3TzTE+UXPhV55QoABhjXjfGDDTG9DPG3J1te5LFfnDdz5+x/Ke1AX4Ia7/cHxaPcFqmufD0RPC9p+dyyaP+bqLNu6uYvjjYvspe2J+90FVDt2/Z0Nq7ZHSvmO6cYmuZZ78ZqbHwE+Eg32FTxl0sl4zuxQc/Pjmh60T2FGI93hXW3gvnHBG9FWg8RN6jZ7vSqN3UWhU3xB7sXoH7Mxe7lg5PhetH3UfNgIbBQg1fdq0VVWvM9bB1TzXH//od7n59sevc3O0pNMboe6Zz9ZOzEp6JaveS3O6jsuL4B9AlssCYX3k3xfWMgnLqkM4pew7jGYfRqXUJy+8+nctGxxcTiGTCYQ29gl9dMDxqcb9/fnM0U38QPRfAb1RQw45w4c9EY09IWXGB0wvOhZ91rg1JbXLUe/QU7Irdq5U5ZcFGThzUiZLCfHYdqAHg3aVb4Wzreo4opNFoDxas30WeCIs27mZ49/JAY6Q/Xf2l5+qX9Qbsen3B+l2BbbBbk+7x5q1bFPL2TSewylr10m9Y4Y0TBwa6x+E923gu1+1X3s25p1BUkJcyUdi2tzrsfWOlmor5ISN6tGH1vWd6HjtzRFeO8xnZFvksTBzSmTYtCunUuhgg7mHSn9020Xlu3ZfOlkCoKKQZe6id+8djVySRDeZZq7/kun98xtfH9OYX5w1zKlR3xeMEmjPwxBhjGHzbFE4f1oWX5m5w0luVFDD/jtN8z1u1bR99OrTkogc/9jxeU1dPfl4+7y3byuWPfxLYHlsQ3SOBurVpQZ8OLenbsczK433uwM4NIharo+JXqvY9x/ZtT6fWxbxslceGnQcCWt806FBWxLa9B4GQ6yRVj+G3T+jHna82zGXIgQazPxHGPXL5SOd11/ISrh7Xh0rX5MrGOsZFrrkuQcrz9GFduOLYiiCWJoS6j9LMQ++FZiu6v+w611ZgNa5+8479oZ6BXdHYudzd0UzOU6g3oWGfbkGAUEvogr985HvelX/9JKaL6OH3VgJw/VOfxWWPXVTuCT5dy0vC8lTXes+HmDSsS1RapOupX8eWvuVqi3qPti3CROlHz33ua2+kbYcaI3u3jUprWxqK4fRqV8pNpw50esItYgwhDUJkJZct33qQfl+sSWUf3zKBbx7fsFlPWXEBhXFsIORWHL+f0E/OOIwxfdO3L7WKQpp5f3loNI5dqXyxfR9z1jYMz3zwvyuYsmATG3YecPYStoe82QHmugz3FGrr6hl+x1SembXWN8/sL3aw/6B3N/lgbT2Pf7ja99xH3w+JQnVtfBP6fj9tGWu27w9rWUX6gb0myXVqVRz23i66s0Z0ddKO7dee6Ted2Og+uyKw7suG3oFfGUDseMfiOyf5HgO4+JieMY9ngkvHRPvsO5SFyvLJb4yiW5sW7LIaMt3bJDbqyCZPwlvMseYpZIJYv64gDbJRfdoBoXIaN6ADl4/tHey+Aa6dbr1U91GGsL/IE+77b1j66ws2sfit8PkK9kNn+8ndIzPqXK6n6tq6sNEPfnxUuY3WLQoZ1r08kK37quvYU1XLLS/Mj5lvy+5qKjpEP0KlRfm8NCdqzqHD7qpa9lTVRC2AF0l5i0InrmIzdeEmauoMpUX5PPmNUdG2u0ThmuP78Mj7q1KyyYnbhe1eoTZWSCHWbRtrWV9/Un+e/tRflBPh2evG+rr0vHC3VF/8zrEM6NyK6po63ly0mT4dWgJwdO+2XHlsBd85sZ/PVYIhIrQqLmB7bcg19bVjetK6pJBubVpw28sL+HLfwaSun0qCNMi+clQPjuvfgc6tQ73FO88dxt8+/qLR89zPX7ZiCtpTyBB+D9Lijbuj0vZW13L/9OVc+/fZgHdPYdveagb9dArjf/0O89eFB2sXbtjFTNf+rZc8OpOz/vRBHMYGy+ausN3uohVb97FxV2xf+4kR4njXecOi8njt/3uwrp4VW/fSv1MZx1S0izru7il4DSGEBveT2w3kjPRtxH0kCJeNCdbqS4ae7Up9j3335DjG4rs4pqId3QK4tLpYFZk7DtOyuICy4gLalxWHzQQuyM/jjnOG0ql18q6ylq6e1YDOrbhmfF/OHNGV4QEbM+kicrBE0Lq6cwJlEuTa6V42X0UhQ1TX1DHop28Eyvth5XZ+5+o9eImCzZov9/PoBysb7lNbx5n3f8DXHg6tMJrI8M+g5+xzuU7eWx4+ac0ORvqx3dXy69Sq2LOi9dr/976pS1m1bR+fRwihzUFXjKZ/pzLPPOce2Y0rj63gR6cNjjrW2JaKInDCoGBLq/gV42NXjPQ+EJAbTw2ftXtMRbjvvyjJkTkXjezBp7eeEtazTGZVUYDpN53A36+O7tlNGtoQ63EX/dku197grulZDdQLr2f/s9snMvf2iU6Fnc4WvN+1bztrCIO7tOLqcX0Y2Cm95aGikCFWbN0btw/dxivQ7Mb9HN30zLywY6/N3xj3/WINszxhYEOF+Mb8Tc7rK+IYRRSJvYTwojtP46KjGzYniWcz90j+fvUoDu/RBohufRUX5HPHOUMpL41e1sDvR2n3WkSCB/n9FoJLZG5FLCIfCffKre7JfRA0kAodI+IwhV77V8ZBv45lHD8gWkzvn3wkn902EYBa1wAMd8/6f08dxKv/My6p+8eL+/6tSwppU1rUMOcojargF8S+elwfpnx/PLedNSSuPRcSQUUhDdireUJDZbKvuvFVQv1wi4LXcgDuh+TNReEzhtfviH/IZKwlB8b2axj10Nj2koO7tOLKAEPnzrRahaVFBdzzlYZNUYKsrOqHe+RPkB/xuAGhMel+PQV7zHqPtqWBd+1q4yE60BCH+MPXjuC2s5LfQyry23IHbGdbFW48eJVXflwjaGJjx4KuOq6CooI82lnCZT/nL3zn2LD8hfl5DOteTv9OZfz8nMTWrgqK7bYb3r21b55s9BQyiQaa08DKbQ0btN921hBufXEBo/q0491liS3zfaCmjpfnrqe0qMAZoeSmIE/47r/m8Mq8DVHHEun2x+opfGt8X04b2oWTfvNfJ23zbu/tLkuL8uO+f2F+Ht3btGD9zgNJ+U5FBJHgrrNvWK36DT6xkEnDunD8gI6cPLhT4KCn32xn20VhL6nwi1eD7TXQpXVJ2Fo7fni53Rru3fh9vCqmZHptkZwwsKPnpDH7ufNbP2najSekzAY/jurVlinfP97TRZMD9XVGUFFIA+7VFksK8ikqyHOWtkiU7z091/dYfl4er8yL3t1qw84D3PVaaImMWJvBTFmwib4dWzqBxVhLN4gIfTq0pENZMScO6siPnpvHM7O8d9ZqWVyQkCh9d0J/fvz8fPYn0VMQ4vsR294Rv9mo+SJMHNI59DrgZ3KXY1F+nhPvSHQO9IyfTPBMd/vBrz+pH6u372d9EpPqvFwYqRQFP+wGT0GSrqpkGdzFu5eQ7Nz1IJtCxRL0TKHuozTgbmnl5YV80B9Wbvc/IUn8frDH3vt2zPPq6w0LN+ziun/M5tTfv8e6HfsBf1F46frjnNfFBXks3bTHVxAAWhTmJ7QMwpCuoQDn8B6JjzoJ9RSC57ft9J285joQpH4c0aM8rMflXhYkcs2rebefyqyfnuK42uLdzc19tR+eNjimGAbZDN67p5D+qqKX5bopaqK72b3/o5P44McnxczTv1MZrT02AMokTbP0s4y7U5AnEtgHnSiJjgz5+4wvOPP+hqGq4371DuDvPnJvYJOXFz4b24sNuw7Q2ECYa46PDsYO71HO2zedwLdP8B/7PrpP9HBUN+L6Nwi2KPh9V3lhQt/4df/xzdFhrr7Duraib8fQ2P5IF055aSEdyoqd7SvjbZVHXi/ZFq3X3VMZU/DjsSuP4ZHLR6Y8EJ8qki2BNqVF9GjrP9TY5sKjsztxUUUhDbhbgnkSfDPuw7r6B7diEVRztuypYujtU5xF6Dbu8o4FBNk4Jk+ElVv3xcxTkJfXqCC2Lyv2TO/bsYxCnxbjx7eczL+uGRPzuiIN5RKkfPKcvA2Z3YFi93cYRORblxSGrfw5eVQvZ9avX/lmwmcd66s9slcbAM9yz8SyKh3Kih0XXXPmfCvWdHKAvR3SgYpCGnD/8PLicGNMPCyxhyDoVIR3l25l38E6Hv8gtLmP305TQXZJyxcJmxMAMH5g+JDDlsX5Tqv68J5twoab2px/pP+a+O6KeIBrzkHHsuJGW+sS57bnXiNuCsJcRt6vY9GzXUgEfnHeMI7s1bZhc5g47ApC1PUSvIE92quFh1+7JMDM+abOHy4+gsN7tqFlUXp7MsN7lLP63jOdRR4zjYpCGgjvKYSWdQjC1a6FtDKBV912538WNbq8he+55wwNW/yrKL+hpzCuf3vuu+hwLjiqQRhunDgw5qxPd6Xs3gMhyH4IbvtS4b1zu+iCutePH9CRV/9nnLPuv30Jv8mBCdsZcb1YcYNYelFV4y8K6R4bfygw4bDOvHz9cUlP5Mt1VBTSgFsUjuzV1jdf5NZ/8T5r9sSkIDOQq2vrG0akCNzxykLufWNJVL7HA24RusLDdVRSmB/W4i4pbOgp2HGKu89vWM6isYBiP1dLqW1pYdyLrjnLXKTAMRM2eMCn9vaaYDWse7ljh12Z+A1ES9TOyG8/aM/xX9eMCZsnYY/2KolYl2lwgL0zlKZDbkZ0DnHsOO2T3xhFl/ISzhzeNWpmcYvCfB678hgqbn7NSYt3pM6bPxjP0XdNC+wt+MO05QC88Jn/YnXJkJ8nYcJWUpjvtDqra+qdtBm3TOC7/5rj6U5y06W8hGV3nc7//beSyaN60aqkgJraYJ9WJDEfve0SO6aiLau27XfSg8QUWpUU8MJ3jo2aRWxz13nD6dhqme8yGYn2FKICzTGKyH1sZEVbxvZrz5QFG/l09Y7QGP2Fm8JcdfPvODUlG9oohw76bacBu+VuV5Df9lhB0qsLGm+l0L6smLalhYFbhummVUlBWOV52ZjezhLI+1wT0bqUl/DMdWN9g8xuigry+P4pITdTaVGB59IUXiS6FEG15UL5w8VHhqW7v69Yi+Yd1astvdu39DzepbyEX35lRMor2Uh3UZBhp15cdVwFH/z4pLABD61KCnNi7LySOVQU0oDdU7DdAUEFILKnMK5/B1o2ssRyJjYjuWxMr7DdpbxYdOdpIXeRy56je7d1lojeX5P4RLREcJdKPEVkr08VOSzS/RWmq8xTfdUHLzsq0P3sDW76dyoLNGRSadqoKKSB+oiegtcQxEih+MxjjZoLju5OWYCJLIm2DP3o3qYF824/1Xl/13nDGx0qaI9Oiawvh1itzrFp3CnKi0TrbXuHtmhRaPyCSWtFBtxHYbezDD5rRDdW33tmoJ6b0vRRUYjB2F9O528fr46Z59+frqHi5tfCFm9rWJs/9KPzcidE+qXbtSyKqniMadiC0m9NeSF4JdAYti98/c4Dgd00NnZAOVLsBnRuxeyfnsKlo6N38UonggSSyl4R+xbcd+HhfPKTCVGfI0jvINkeRDyB5u5tWjhLg/tPXvO6Xo74GpWcRUUhBht3VXH7ywtj5rl/eiUQ2vTGJjKmUGZtUOJmQOfoMcheo4/sXsapPi11kdT9zE8clPxkmW7l0SOE2pcVZ3zPXZGG7yHWnV++/jje+N7xzvuigjxnw5iwYa1B7pmAnYny4c0nc6+1oqzf6COvInc3Lpr2wEolUVQUAhBrH1670na3LO2YQl5YWvhP96HLon30XhWnfd7k0b144qpjnPRTnIlukrKegh0U/tGkQY3kbOD5b49l2V2nO+//fMmRMXJnjvCYgn/117Zlke9M8lNckwkjL/HnS45kYISwJ6t78Z7fWH6vw3++pCHOkAvLNCu5h4qCD2530Nw1O6OOG2MwxrjiB9EC4G75uyvu3u1LPd0zXj0Fe3G6wvy8sJb8/ZNDlW/otqlRhUvH9OK+C0fwrfHB99s9une7sPkG7XyGY2acFPSg7jx3mO9cirNGdHP2KW64ZbLuI2/atyzynH1ui130PBX/T94yR9cVUnIHfUJ8+OlLC5zXXi3NPre8zvEDOjT0ClxZbFEQD6GA0DINXnjdZ3Sf9kxbvDlq5Ux7y8VUxhQK8/O4aGRyi3Fl2k3kh7uCTtSiwvw8OrcuZu2X3stQR4pAshNd/cruk1tP8bm/N5ExrXjvpzRvtKfgw6pte53Xkx+Z4Znn/eXbGlppElom4KMV25wfpbv34F5c7S+XHR3Yjj9NPpK3fjA+aqy4vdRDKn/XqVjNNVfqmVBMIYXX86iCoz5rmj57fp54Dmt21lLyCTTnyFehHGKoKPgQtD6xewofr9jO//xrDpc8MpNlm/cA4S3HZ7411nkduf9tLFoU5TOgc+xlBlJV+aViTZdE9k9IB+L7Jj7iKdt0uY9889trKUVOXnN6qkmZozRTVBQiqK83VNz8GnMi4gh/n/EFlVv2Ru1LbLuFvvf0XN6y9kfeXVUDhFeQfrNcJ4/q5biCEiE09DI1qpCKSiRX1gpLtTh5XS4yLdOB5kZ7CjnyXSiHFhpTiOD305Z5pt/20gLKigvYW13LJaMaxtx7bXL/wDsrgOgf5a8vHEFFhDj88ivD+aVrs3o3AxvpIdj3iKc126a0kJ37azyPxdNTeON7xzurarqxK6psV0juAHwypgzt1pp1Ow4EWuohV+rgBo9mrlikHEqoKEQwffEW32P2RvIHaxuWuYxVIUe2Vr8aRxB39k9P8Zxh+tx1Y1m5rWGFUiG+UTaThnbh6U/Xeh6LJ6bgN4zT2dgmDpvSgbiG6iYTUP3dV49g0cbdgVx+mZy8Bq6eQkS68z7bX4JySKLuowj8dvty8+KchlVG62KoQjIuDL8lB0ZWtAsTl+rael79fIP3NVoWMapPu7B1i2Lt/5uK0Sh2xZb1kS0pun3L4gKOqfDe+jOyEk/6lgnOU/DdnyFJc5TmifYUIigO4N//yYsNm9D47WcMmfGvb9930PdYQb7wzLfGUl3b4OaJVVmnItBsX2Nk77ZJXysZUjnTOyjJxjESjilEpAfZX0NR/FBRiOCT1V/Gld/tSookW63lCYM7MX3JFmfiVaFrq7BYFVek++jf144JtCBf2DXyhNe+O843sJ4phNBS3uC/blSyTB7VK3yfjAx/3X5f5R3nDOUXry5ijLUIYX6eOJMgFaUx1H2URhJteJ81omtS9/3dV48A4JJRvUN25Ak3nNQfgFgdIYk4Nrpve4Z2i79CHdqtPGqtp0wjInQtb8GL3znWN5CfLOMGdGD1vWe67pnc9eI93Xm+Iur7fh3LeOKqUU5wfM7tE5l7e/QqvIrihfYUXKS6252IO2H53acnPYmsvLSQyrtPD99XWBq3KRWT13IF+5PE2g41XfdM+Py4y9/bfRRJ65L4VrxVmjfNtqdgjOHh91awZU+Vk/b6/E0pvUciolCYn5eSTdIL8vPCKhlnv+IYNuXKxLNUkI2Pkqy7MN5Gif2YaAxBSSVpEwURuUNE1ovIXOvvDNexW0SkUkSWishprvRJVlqliNycLtsAFm/cwz2vL+GGf85x0nbs9w/aJkIu1bFB5g/kNaEmQjbG6Gf6juITaFaUZEi3++j3xpjfuBNEZAhwMTAU6AZME5GB1uEHgInAOuBTEXnFGLMoHYb977PzgPDAcmkjW1/GSypa/KnCNiWWRU3KfZSVnkKy58d3ASekoKqgpJBsxBTOBZ42xlQDq0SkEhhlHas0xqwEEJGnrbxpEYVFG3c7r2vr6skT4e0l/hPXEiGXqtg91sS7WEsnNyX3UTbIdPnp16Wkg3Q7DG4Qkc9F5HERsSN+3QH3lNp1VppfehQicq2IzBKRWVu3bk3ayNp6w8vz1vPq5xsbzxwHuVTJrtuxH4Ce7Urp5DM7N5d6NsmSQ0WfMO/874lhEw8jESfQrF0FJXUkJQoiMk1EFnj8nQv8BegHHAFsBH6bvLkhjDEPG2NGGmNGduzYMenrHayrZ1919Do+yZJLdWxNXajiKC7Ii1qf/+7zh7Hql2d4nXbIkpWYQopv2adDSyb6bMPqvp+6j5RUkpT7yBjjvftHBCLyCPCq9XY94F4EqIeVRoz0tFJbZzx3tkqWrC/14MKevFTgoVSXju6daXPSTlZiCllyGKooKKkknaOP3DOwzgfsrcxeAS4WkWIR6QMMAD4BPgUGiEgfESkiFIx+JV32uamtqw+0Cma85FJPwV6OIxVLWRwKZONTJitEdqA/27PBleZNOgPNvxaRIwiNmFsNfAvAGLNQRJ4hFECuBa43xtQBiMgNwFQgH3jcGLMwjfY5HKzzX6oiGXIpplBXH/qMBU1p3GkMstFLS/aOLYryeeyKkRzes02w++k8BSUNpE0UjDFfj3HsbuBuj/TXgdfTZZMftXXp+VHlkijYMQXtKaTxnin4vicc5h9D8LufSoKSSppHs7ERaurq09LailxLKJvYMYXC/PCKy712T1MiG3qcab1tHvKuZBpd+4hQK7o+DaKQSz2FyJjCtBvHZ9OctJNJ91HX8hI27qrKuMtKRx8p6UBFgVBPIR1hhVzy1ETGFPp3anyrTyUYz3/72Kg9vTOBe57CHy8+gtZpGEGnND9UFAjtnhZrB7VEyaWewj3nD+feN5YwsEtZtk1pcnRr04JubVpk7f7GwLlHeM7zVJS4UVEA6utNemIKuaMJjOjRhqeuGZNtM5QU4riPsmuG0sTIoVBo9qg3pGVnqlzqKShND10QT0kHKgqEBEFFQTnk0MdLSQMqCoQm/6Rn9FHKL6koDnagOXKYsaIkg8YUsALNaRh9lEtrHylNjw5lRXz35P6co0FmJYVoT4FQTCEdPQUl8wzo1HxGV4kIN546iP7N6DMr6Ud7CoRGH6koNA2eu+5Y1u88kG0zFOWQRUWB9AWalcxTXlpIealO4lKURFH3ESHXkYqCoiiKigIQEoWaNK2UqiiKciihokAo0FxVk/x2nPPvOJWh3VqnwCJFUZTsoKJAKKZQXVtPUX5yxdGqpJDSotTv4KYoipIpVBQIuY+qauooLvQvjq8cFWwseLb26VUURUkFKgqERKG6NvY+zb+58HCevW5s4xezNKE5jZdXFKXpoKIA1NdDdU0dxQXhxTFpaBfndV6eBHIv2f2En587NJUmKoqiZASdpwDc9Ow8APp0aBmWnsgqFeXWRieFScYnFEVRsoHWXDFIZJXTX10wgp+cMZiRvdumwaJorjuhH4X5wo8nDc7I/RRFadqoKLiIkoCIhPwAy562bVnEteP7ZWwxvJtPH8zyu8/g2yf2y8j9FEVp2jRbUfhFAJ9/ZLU+tFtrfnjaoPQYpCiKkgM0W1G4bExvnvrm6PDECBWIbO2LCNef1D8sbXj38nSYpyiKkhWarSiICIO7xp59HMQBdOaIrqkxSFEUJQdotqIA0ZV+1HsfVfjXNWPo1a40HSYpiqJklWYtCo2NLvI7PrZfe04fFprDoNswKIrSlGjWotBoDCGOcxVFUZoCzVoUGhs1etbhjccLDNpVUBSl6dCsZzRHuocEePraMazYupdLR/empq4+O4YpiqJkiWYtCl4dhTF92zOmb3vf4w3nqv9IUZSmR/MWhaiYQuT7xit+DTQrhxozbpnA7qqabJuh5CjNWhQaG30U62jn1sUAdCgrSqFFipJ+upSX0KW8JNtmKDlKsw40RxLpEoqlGZePreBPk4/koqN7ptkqRVGUzKE9hRjEch/l5wlnH94t1SYpiqJklWbdU2gspqAoitLcaN6ikG0DFEVRcoxmLQqJbKKjKIrSlElKFETkIhFZKCL1IjIy4tgtIlIpIktF5DRX+iQrrVJEbnal9xGRmVb6v0Uk7cN6EhmCqiiK0pRJtqewAPgK8J47UUSGABcDQ4FJwP+JSL6I5AMPAKcDQ4DJVl6AXwG/N8b0B3YAVydpW6OoCCiKooSTlCgYYxYbY5Z6HDoXeNoYU22MWQVUAqOsv0pjzEpjzEHgaeBcCdXOJwPPWec/CZyXjG2JUF+f2Ey0Ph1aptgSRVGU7JCuIandgRmu9+usNIC1EemjgfbATmNMrUf+KETkWuBagF69eqXIZKitT2yto9e+O46qGl0nSVGUQ59GRUFEpgFdPA7daox5OfUmNY4x5mHgYYCRI0embKGJugR7CqVFBZTqxGZFUZoAjYqCMeaUBK67HnBP9e1hpeGTvh1oIyIFVm/BnT9j1CYoCoqiKE2FdA1JfQW4WESKRaQPMAD4BPgUGGCNNCoiFIx+xRhjgHeAC63zrwAy3gvx6in86oLhvPmD8Zk2RVEUJSskOyT1fBFZB4wFXhORqQDGmIXAM8AiYApwvTGmzuoF3ABMBRYDz1h5AX4M3CgilYRiDI8lY1siePUUvnZMLwZ2bpVpUxRFUbJCUoFmY8yLwIs+x+4G7vZIfx143SN9JaHRSVkj0ZiCoihKU6FZz2iOpFZ3WlMUpZmjouBCewqKojR3VBRc6OgjRVGaOyoKLrSnoChKc0dFwYX2FBRFae6oKCiKoigOKgqKoiiKg4qCoiiK4qCioCiKojioKCiKoigOKgqKoiiKg4qCoiiK4qCioCiKojioKCiKoigOKgpNjMJ8ybYJiqIcwiS1n4KSWzx73Vi6t2mRbTMURTmEUVFoQhxT0S7bJiiKcojT7N1HN00cmG0TFEVRcoZmLwr/M2FAtk1QFEXJGZq9KAD8+oIR2TZBURQlJ1BRAI7o1SbbJiiKouQEKgpAno7iVBRFAVQUABBRVVAURQEVBQBUEhRFUUKoKAB52lNQFEUBVBQAFQVFURQbFQVANUFRFCWEigIqCoqiKDYqCujoI0VRFBsVBXSegqIoio2KAhpoVhRFsVFRQOcpKIqi2KgooDEFRVEUGxUFNKagKIpio6KA9hQURVFsVBTQnoKiKIqNigLaU1AURbFJShRE5CIRWSgi9SIy0pVeISIHRGSu9feg69jRIjJfRCpF5H6xamQRaScib4nIcuv/tsnYFt/nyNSdFEVRcptkewoLgK8A73kcW2GMOcL6u86V/hfgGmCA9TfJSr8ZmG6MGQBMt95nBJ2noCiKEiIpUTDGLDbGLA2aX0S6Aq2NMTOMMQb4G3Cedfhc4Enr9ZOu9LSjkqAoihIinTGFPiIyR0TeFZHjrbTuwDpXnnVWGkBnY8xG6/UmoHMabQtDewqKoighChrLICLTgC4eh241xrzsc9pGoJcxZruIHA28JCJDgxpljDEiYmLYdC1wLUCvXr2CXtYX1QRFUZQQjYqCMeaUeC9qjKkGqq3Xs0VkBTAQWA/0cGXtYaUBbBaRrsaYjZabaUuM6z8MPAwwcuRIX/EIioqCoihKiLS4j0Sko4jkW6/7Egoor7TcQ7tFZIw16uhywO5tvAJcYb2+wpWedtR9pCiKEiLZIanni8g6YCzwmohMtQ6NBz4XkbnAc8B1xpgvrWPfAR4FKoEVwBtW+r3ARBFZDpxivc8IKgqKoighGnUfxcIY8yLwokf688DzPufMAoZ5pG8HJiRjT6KoJCiKooTQGc1oTEFRFMVGRQFd5kJRFMVGRUFRFEVxUFFQFEVRHFQUFEVRFAcVBUVRFMVBRUFRFEVxUFFQFEVRHFQUFEVRFAcVBUVRFMVBRUFRFEVxUFFQFEVRHFQUFEVRFAcVBUVRFMVBRUFRFEVxUFFQFEVRHFQUFEVRFAcVBUVRFMVBRUFRFEVxUFFQFEVRHFQUFEVRFAcVBUVRFMVBRUFRFEVxUFFQFEVRHAqybUCu8LuvHk7X8hbZNkNRFCWrqChYfOWoHtk2QVEUJeuo+0hRFEVxUFFQFEVRHFQUFEVRFAcVBUVRFMVBRUFRFEVxUFFQFEVRHFQUFEVRFAcVBUVRFMVBjDHZtiEpRGQr8EWCp3cAtqXQnHSQ6zbmun2Q+zbmun2gNqaCXLOvtzGmY2TiIS8KySAis4wxI7NtRyxy3cZctw9y38Zctw/UxlSQ6/bZqPtIURRFcVBRUBRFURyauyg8nG0DApDrNua6fZD7Nua6faA2poJctw9o5jEFRVEUJZzm3lNQFEVRXKgoKIqiKA7NVhREZJKILBWRShG5OUs29BSRd0RkkYgsFJHvWentROQtEVlu/d/WShcRud+y+XMROSpDduaLyBwRedV630dEZlp2/FtEiqz0Yut9pXW8IkP2tRGR50RkiYgsFpGxOViGP7C+4wUi8i8RKcl2OYrI4yKyRUQWuNLiLjcRucLKv1xErkizffdZ3/PnIvKiiLRxHbvFsm+piJzmSk/bb93LRtexm0TEiEgH633GyzAhjDHN7g/IB1YAfYEiYB4wJAt2dAWOsl63ApYBQ4BfAzdb6TcDv7JenwG8AQgwBpiZITtvBJ4CXrXePwNcbL1+EPi29fo7wIPW64uBf2fIvieBb1qvi4A2uVSGQHdgFdDCVX5XZrscgfHAUcACV1pc5Qa0A1Za/7e1XrdNo32nAgXW61+57Bti/Y6LgT7W7zs/3b91Lxut9J7AVEITaztkqwwT+kzZunE2/4CxwFTX+1uAW3LArpeBicBSoKuV1hVYar1+CJjsyu/kS6NNPYDpwMnAq9YDvc31w3TK0voRjLVeF1j5JM32lVsVrkSk51IZdgfWWj/6AqscT8uFcgQqIirduMoNmAw85EoPy5dq+yKOnQ/803od9hu2yzATv3UvG4HngMOB1TSIQlbKMN6/5uo+sn+kNuustKxhuQiOBGYCnY0xG61Dm4DO1uts2P0H4EdAvfW+PbDTGFPrYYNjn3V8l5U/nfQBtgJ/tVxcj4pIS3KoDI0x64HfAGuAjYTKZTa5VY428ZZbNn9L3yDU8iaGHRm3T0TOBdYbY+ZFHMoZG2PRXEUhpxCRMuB54PvGmN3uYybUdMjKuGEROQvYYoyZnY37B6SAUPf9L8aYI4F9hNweDtksQwDLL38uIQHrBrQEJmXLnqBku9xiISK3ArXAP7NtixsRKQV+AtyebVsSpbmKwnpCPj+bHlZaxhGRQkKC8E9jzAtW8mYR6Wod7wpssdIzbfdxwDkishp4mpAL6Y9AGxEp8LDBsc86Xg5sT6N9EGpVrTPGzLTeP0dIJHKlDAFOAVYZY7YaY2qAFwiVbS6Vo0285Zbx8hSRK4GzgEst4col+/oREv951u+mB/CZiHTJIRtj0lxF4VNggDX6o4hQMO+VTBshIgI8Biw2xvzOdegVwB6BcAWhWIOdfrk1imEMsMvV1U85xphbjDE9jDEVhMrobWPMpcA7wIU+9tl2X2jlT2tL0xizCVgrIoOspAnAInKkDC3WAGNEpNT6zm0bc6YcXcRbblOBU0WkrdUjOtVKSwsiMomQO/McY8z+CLsvtkZu9QEGAJ+Q4d+6MWa+MaaTMabC+t2sIzSYZBM5UoaNkq1gRrb/CI0EWEZoZMKtWbJhHKHu+efAXOvvDEL+4+nAcmAa0M7KL8ADls3zgZEZtPVEGkYf9SX0g6sEngWKrfQS632ldbxvhmw7AphlleNLhEZw5FQZAj8HlgALgL8TGiWT1XIE/kUoxlFDqPK6OpFyI+Tbr7T+rkqzfZWE/O/27+VBV/5bLfuWAqe70tP2W/eyMeL4ahoCzRkvw0T+dJkLRVEUxaG5uo8URVEUD1QUFEVRFAcVBUVRFMVBRUFRFEVxUFFQFEVRHFQUFEVRFAcVBUVRFMXh/wGys4ZTx133AAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 使用Matplotlib绘制奖励值的曲线图\n",
    "plt.plot(episode_rewards)\n",
    "plt.title(\"reward\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "fa8efb5e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAM8AAACeCAYAAACVU14NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8+yak3AAAACXBIWXMAAAsTAAALEwEAmpwYAAARfUlEQVR4nO3dfXBVdX7H8ff35uYGyCN5AEISnoQAgRGCoOLTMoWVSKvBkUIYIYCIKQXErWu17Uy37bSd7tRZt1pYhbqu6VBdy259oFVH0HEZWl0FFOQxAcUlYuQhRJ6WQPLtH+eEvWTzcO8hN/ch39fMmZz7O+ee8/vB+eT87jk3vyOqijEmfL5oV8CYeGXhMcYjC48xHll4jPHIwmOMRxYeYzyy8MQIETkrIiO6a10RmSYiR7undiAiKiIju2t7icAf7Qr0RiLyBTAQaA4qLlbVr0J5v6qmRaJeJjwWnui5W1U3R7sSxjvrtsWI4G6RiPxMRNaIyH+LyBkR+VBErutg3Vkistddr05Evt9mu4+KyDcickxElgSVp4jIkyLypYjUi8izItI3aPlj7nu+EpEHIv8vEH8sPLGrAvhboD9QC/xDB+s9D1SpajowHng3aNkgIBMoAJYCa0Skv7vsn4BiYCIw0l3nrwFEpAz4PvBdYBQwo7salUgsPNHzqoicdqdX21n+X6r6a1W9DGzAOcjbcwkoEZEMVW1Q1R1tlv2dql5S1f8BzgKjRUSAh4DvqeopVT0D/CNOYAHmAi+o6meqeg74m2tsa0Ky8ETPbFXNcqfZ7Sz/Omj+PNDRRYL7gFnAERF5X0SmBi076Yav7XbygH7A9tYAA2+55QCDgd8Eve9IiG3qVeyCQZxT1Y+AchFJBlYCrwBFXbztBHABGKeqde0sP9ZmG0O6o66Jxs48cUxEAiJyv4hkquol4Fugpav3qWoLsB54SkQGuNsqEJGZ7iqvAItFpERE+gE/iFAT4pqFJ/4tBL4QkW+BPwHuD/F9j+NciPjAfe9mYDSAqr4J/Bjn4kMtV1+EMC6xP4Yzxhs78xjjUUTCIyJlInJARGpF5IlI7MOYaOv2bpuIJAEHcW6wHQU+Auar6t5u3ZExURaJM8+NQK2qHlbVJuBloDwC+zEmqiJxn6eAq2+wHQVuaruSiDyEc5cb4IYI1MOYbqGq0l551G6Squo6YB04X3SMVj2M8SoS3bY6rr47XeiWGZNQIhGej4BRIjJcRAI4XzZ8PQL7MSaqur3bpqqXRWQl8DaQBPxUVfd0936MibaY+IaBfeYxsayjCwb2DQNjPLLwGOORhccYjyw8xnhk4THGIwuPMR5ZeIzxyMJjjEcWHmM8svAY45GFxxiPLDzGeGThMcYjC48xHll4jPHIwmOMRxYeYzyy8BjjkYXHGI+6DI+I/NR9IOxnQWXZIvKOiNS4P/u75SIiT7tjVO8SkUmRrLwx0RTKmednQFmbsieALao6Ctjivga4C+cBsKNwRgP9SfdU05jY02V4VPVXwKk2xeXAi+78i8DsoPJqdXwAZIlIfjfV1ZiY4vUzz0BVPebOfw0MdOfbG6e6oL0NiMhDIvKxiHzssQ7GRNU1D3qoqupl3DUbq9rEO69nnvrW7pj78xu33MapNr2G1/C8Dixy5xcBrwWVV7pX3W4GGoO6d8YkFlXtdAJeAo4Bl3A+wywFcnCustXgPEU5211XgDXAIWA3MLmr7bvvU5tsitWpo+PWxqo2pgs2VrUx3czCY4xHFh5jPLLwGOORhccYjyw8xnhk4THGIwuPMR5ZeIzxyMJjjEcWHmM8svAY45GFxxiPLDzGeGThMcYjC48xHll4jPHIwmOMRxYeYzwKZazqIhF5T0T2isgeEVntltt41aZXC+XMcxl4VFVLgJuBFSJSgo1XbXq7UIaGajNM1GvAd4EDQL5blg8ccOefA+YHrX9lPRt6yqZ4nDo6bsP6zCMiw4BS4EOucbxqG6vaxLuQwyMiacAvgEdU9dvgZeqcPjScHavqOlWdrKqTw3mfMbEipPCISDJOcDao6i/dYhuv2vRqoVxtE+B5YJ+q/ihokY1XbXq1LofbFZHbgK04Y0+3uMV/ifO55xVgCHAEmKuqp9yw/SvO0+TOA0tUtdPPNTbcrollHQ23a2NVG9MFG6vamG5m4THGIwuPMR5ZeIzxyMJjjEcWHmM8svAY45GFJ8aMGwfDhkFaGki7dxciLy0NJk2CvDxISYlOHeKB3SSNIUlJsH49XHcdnDoFX3wB+/bBoUNw+DB89RVcvBj5eowbB+vWwfnzzj6PHIE9e+DgQTh2DOrrIQYOmx7T0U1Sf09XxHROBPr1c6b8fOdAPnYM9u+Hl1+Gzz/vmXokJUFWFmRmOmGeNAlqa2HnTqcePRHiWGfhiTHHj2ewadO31NQ4gTl6FBobe/pg7cvu3c3s3t3EwYPOGbCuzjkTNTf3ZD1im3XbYojf72fatDvYsuVdz90in89PIJBKR39epdrCxYtnO91GSclYzp07w5EjR71VIsFYty1OqMo1fZ4YPWIG0254FJ8E/9fqlSidPvclr77zPc7/tqGTrYidYUJg4UkwA7JHk5cxBp8ktbu8X6A/fftmdREeEwq7VJ1AfD4/+XnXI538t6Ykp5OXPaoHaxU9WVlZjB8/Hr8/MucIC08CCSSnktYnt9N1knwBMtN/bzyWhDJo0CBWrlzJ+++/z7Zt26iurmbKlCkkJbV/NvbKum0JJBDoR3q/fKSTu6tCEtkZw3quUj1ERBgyZAgLFixg0aJFjBgx4kpYKioqKCsrY+PGjTz99NPs3buXlpaWLrbYNQtPAigYOJEBecXU1e/s8LNOMBEfRYMnkZ01lF37XkP12g+k321bSE1NpaioiIaGBo4fP05zBK8++Hw+xowZwwMPPEBFRQWDBw++Uo/gOvXv358HH3yQe++9lw0bNrB27Vpqa2uvKUQWnjjnkyQmj7ufvOwx1NV/0uX6rQfVkPybmFg8l8Nf/i9nztZfUx1SUlIoLCxkypQpTJs2jalTpzJs2DAaGxvZvn07mzdvZuvWrRw6dIjz58/THbdHkpOTmTBhAlVVVcyePZucnJxOz7jgtD03N5eHH36YuXPnUl1dzfr16zl8+LCnOnUZHhHpA/wKSHHX36iqPxCR4cDLQA6wHVioqk0ikgJUAzcAJ4F5qvpF2DUzIUlPHUjBgFL2H3mbCxcaOXXucy40tV5Ja/9gutDUwJG6D5hSsoghg6ew5+CmsPbp9/vJy8tjwoQJTJs2jdtuu42xY8eSmZl51eeKjIwMioqKKC8v5+zZs9TU1LB161Y2b97Mjh07qK+vD/uslJKSwi233MLy5cuZOXMm6enpXYamLREhPz+fxx57jHnz5vHcc8/xwgsvUF8f5i+REIbXFSDNnU/GGTXnZpyRcyrc8meB5e78nwLPuvMVwM9D2EfUh1SNhcnv9+v06dPDek/JyFn6Z5UfaWF+qYokaWrfXE3rl9fhlNo3T1MC6ZoSSNfKe/5D58xcoz6f/+ptlpRoYWHhldc+n0+zs7P11ltv1ccff1zffPNNraur06amJm1padFwtLS06KVLl7Surk7feOMNXb16tZaWlmpaWpq6N8vbndLS0rS8vFzfeustPXfuXNj77Uxzc7Pu379fV61apbm5uSEPtxvuONX9gB3ATcAJwO+WTwXeduffBqa68353Peliu1E/cGNh8vv9escdd4S8vohPZ9z6uC4or9aUlLSw9iUieseUVfrAfRs1I33gVctGjx6txcXFev3112tVVZW+9NJLWlNToxcuXOjWg1bVCdOZM2f0k08+0WeeeUbLy8u1sLBQ/X4n0FlZWbpw4ULdtm2bXrx4sVv33dbly5d1165dWlVVdVWItIPjNqSv54hIEk7XbCSwBvhn4ANVHekuLwLeVNXxIvIZUKaqR91lh4CbVPVEm20+hPMUBXC6eL1WIBBg5MiRzJw5kxkzZoTVDUntm0tKSiqnTh8Je7+p/XJJ9veh8cxXV1008Pv95OfnM3ToUNLS0gDC7hp5oao0Nzdz4sQJdu7cye7duykrK6OkpISkpKQercOnn37K2rVrqa6u5tKlS+3vuKNUtTcBWcB7wG1AbVB5EfCZO/8ZUBi07BCQ28V2o/5bv6enQCCgY8eO1UceeUS3bNmiDQ0N2tzcHNHfrCZ0LS0tevHiRR0zZoxqB8dtWFfbVPW0iLyH003LEhG/ql7m6vGoW8eqPioifiAT58JBrxd8hrn77ruZNGkSGRkZQM/8ZjehExECgQCpqakdrhPK1bY84JIbnL44z+b5Ic4ZaA7OFbdFXD1W9SLg/9zl76rGwFe3o6Q1MHfeeSf33HMPpaWlZGRk4PPZlzviXShnnnzgRfdzjw94RVU3iche4GUR+XtgJ85g8Lg//11EaoFTOFfcepX2ApOZmWlnlwTTZXhUdRfOA63alh8Gbmyn/LfAH3dL7eKIz+dj1KhR3HXXXRaYXsK+YXCNRITi4mJWrFhBRUUFOTk51iXrJSw8HokII0aMoKqqisrKSgYMGGBnmV7GwuNBUVERy5YtY8mSJRQUFFhoeikLTxgGDx7MkiVLWLZsGUOGDLHQ9HIWnhAMGDCAhQsXsnz5coYPH26faQxg4elUTk4O8+fPZ8WKFRQXF1tozFUsPO3Iyspizpw5rFq1inHjxuHz+ayLZn6PhSdIeno65eXlrF69mokTJ0Zs4AiTGOzoALKzsykrK2PlypVMnjwZv99vZxrTpV4bHp/Px9ChQ5k7dy4LFixg9OjRJCcnR7taJo70uvAEAgFKS0tZvHgx5eXlDBo0CLBvNZvw9ZrwZGRkMGPGDJYuXcrtt99OWlqaBcZck4QOj4hQUFDAnDlzqKysZPz48dY1M90mIcOTnJzM+PHjqays5L777qOgoMDu0ZhuFxPhKSgoYN68eRw/fpzjx4/T2NjI6dOnaWxs5MKFCzQ1NdHU1MTly5c7HV8rNTWV73znOyxdupTp06eTkZFhXTMTMTERnkGDBvHkk08CzpgKrWFpamri7NmzNDY20tjYeGUEytbp5MmTnDhxgoaGBiZMmMDixYuZOHEigUDAQmMiLibCA7+72iUi9OnThz59+gCQm9v+wOWtgzC0tLTQ0tJCUlJStw/kbUxnYiY84RIRRMQ+y5iosSPPGI8sPMZ4ZOExxiMLjzEexcqj5M8AB6JdjwjKxRnwPlElcvuGqmpeewti5WrbAVWdHO1KRIqIfGztSzzWbTPGIwuPMR7FSnjWRbsCEWbtS0AxccHAmHgUK2ceY+KOhccYj6IeHhEpE5EDIlIrIk9Euz7hEpEiEXlPRPaKyB4RWe2WZ4vIOyJS4/7s75aLiDzttneXiEyKbgtCIyJJIrJTRDa5r4eLyIduO34uIgG3PMV9XesuHxbVikdQVMPjPjBrDXAXUALMF5GSaNbJg8vAo6paAtwMrHDb8ASwRVVHAVvc1+C0dZQ7PQT8pOer7MlqYF/Q6x8CT6nzUOcGYKlbvhRocMufctdLSNE+89yI82Dgw6rahPOIxvIo1yksqnpMVXe482dwDrACnHa86K72IjDbnS8Hqt3nxn6A82zX/J6tdXhEpBD4Q+Df3NcC/AGw0V2lbfta270RmC4J+peJ0Q5PAfCboNdH3bK45HZRSoEPgYGqesxd9DUw0J2Pxzb/GPhzoPV58znAafdhznB1G660z13e6K6fcKIdnoQhImnAL4BHVPXb4GXuA43j8p6AiPwR8I2qbo92XWJNtL/b1vrY+VbBj6SPGyKSjBOcDar6S7e4XkTyVfWY2y37xi2PtzbfCtwjIrOAPkAG8C843U2/e3YJbkNr+46KiB/IBE72fLUjL9pnno+AUe6VmwDOk7Nfj3KdwuL2558H9qnqj4IWvQ4scucXAa8FlVe6V91uBhqDuncxR1X/QlULVXUYzv/Pu6p6P/AeMMddrW37Wts9x10/Ls+6XWodSCNaEzALOAgcAv4q2vXxUP/bcLpku4BP3GkWTj9/C1ADbAay3fUF5wrjIWA3MDnabQijrdOATe78CODXQC3wn0CKW97HfV3rLh8R7XpHarKv5xjjUbS7bcbELQuPMR5ZeIzxyMJjjEcWHmM8svAY45GFxxiP/h9C+g8ciF0SkQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 216x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 重置环境，开始新的一轮游戏\n",
    "observation, _ = env.reset()\n",
    "# 创建GymHelper对象来辅助显示\n",
    "gym_helper = GymHelper(env)\n",
    "\n",
    "# 开始游戏\n",
    "for i in range(500):\n",
    "    # 渲染环境，title为当前步骤数\n",
    "    gym_helper.render(title = str(i))\n",
    "    \n",
    "    # 找到当前状态下的最优动作\n",
    "    action = agent.choose_action(observation)\n",
    "    \n",
    "    # 执行action，获取新的信息\n",
    "    observation, reward, terminated, truncated, info = env.step(action)\n",
    "\n",
    "    # 如果游戏结束，则结束当前循环\n",
    "    if terminated or truncated:\n",
    "        break\n",
    "\n",
    "# 游戏结束\n",
    "gym_helper.render(title = \"Finished\")\n",
    "# 关闭环境\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab2c6d85",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
